import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler, LabelEncoder
# from scipy import stats

# 读取数据
df = pd.read_csv('../data/水泥煅烧课程项目数据.csv')

# 1. 删除完全空列或唯一值列
df = df.dropna(axis=1, how='all')
df = df.loc[:, df.nunique() > 1]

# 2. 缺失值比例统计 & 删除缺失比例 >40% 的列
missing_ratio = df.isnull().mean()
df = df.loc[:, missing_ratio < 0.4]

# 3. 填充缺失值
for col in df.columns:
    if df[col].dtype == 'object':
        df[col] = df[col].fillna(df[col].mode()[0])
    else:
        if df[col].isnull().sum() > 0:
            df[col] = df[col].interpolate(method='linear').fillna(method='bfill').fillna(method='ffill')

# 4. 删除重复值
df = df.drop_duplicates()

# 5. 异常值处理：IQR方法
def remove_outliers_iqr(series):
    if not np.issubdtype(series.dtype, np.number):
        return series
    Q1 = series.quantile(0.25)
    Q3 = series.quantile(0.75)
    IQR = Q3 - Q1
    lower = Q1 - 1.5 * IQR
    upper = Q3 + 1.5 * IQR
    return series.clip(lower, upper)

df = df.apply(remove_outliers_iqr)

# 6. 编码类别变量
for col in df.select_dtypes(include='object').columns:
    n_unique = df[col].nunique()
    if n_unique < 5:
        df = pd.get_dummies(df, columns=[col], drop_first=True)
    # else:
        # freq_map = df[col].value_counts(normalize=True).to_dict()
        # df[col] = df[col].map(freq_map)

# 7. 特征标准化
num_cols = df.select_dtypes(include=[np.number]).columns
scaler = StandardScaler()
# df[num_cols] = scaler.fit_transform(df[num_cols])

# 8. 保存结果
df.to_csv('../data/cleaned_data.csv', index=False, encoding='gbk')
print("✅ 高质量预处理完成，结果保存为 cleaned_data.csv")
